import json
import os
import threading
import time
from multiprocessing.managers import BaseManager

import facade
from xjlibrary.mdatetime.mtime import MDateTime
from xjlibrary.mprocesspoll.MThreadingRun import MThreadingRun
from xjlibrary.myredis.myredisclient import getDataFromRedis
from xjlibrary.network.MyIP import GetLocalIPByPrefix
from xjlibrary.our_file_dir import BaseDir

curpath = BaseDir.get_file_dir_absolute(__file__)
logspath = BaseDir.get_new_path(curpath, "logs")
rootDir = BaseDir.get_upper_dir(curpath, -2)
sPath = BaseDir.get_new_path(rootDir, "download", "wanfag_cg", "download", "detail")
BaseDir.create_dir(sPath)
configfile = BaseDir.get_new_path(curpath, "db.ini")


class QueueManager(BaseManager):
    pass


class NodeTask(object):
    def __init__(self):
        self.register()
        # 连接到服务器，也就是运行task_master.py的机器:
        self.server_addr = '192.168.30.171'
        print('Connect to server %s...' % self.server_addr)
        # 端口和验证码注意保持与task_master.py设置的完全一致:
        self.m = QueueManager(address=(self.server_addr, 5000), authkey=b'abc')
        self.task = None
        self.result = None

    def register(self):
        # 由于这个QueueManager只从网络上获取Queue，所以注册时只提供名字:
        QueueManager.register('get_task_queue')
        QueueManager.register('get_result_queue')

    def conn(self):
        # 从网络连接:
        self.m.connect()

    def set_task_result_obj(self):
        # 获取Queue的对象:
        self.task = self.m.get_task_queue()
        self.result = self.m.get_result_queue()


class DownDetail(object):
    def __init__(self):
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(configfile, "db", logger=self.logger)
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Host": "www.wanfangdata.com.cn",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
        }
        # self.node = NodeTask()
        # self.init_node()
        self.proxysdicts = ['192.168.30.176:8231',
                            '192.168.30.176:8207', '192.168.30.176:8076', '192.168.30.176:8030',
                            '192.168.30.176:8140', '192.168.30.176:8012', '192.168.30.176:8033',
                            '192.168.30.176:8171', '192.168.30.176:8031', '192.168.30.176:8160',
                            '192.168.30.176:8165', '192.168.30.176:8041']

    # def init_node(self):
    #     self.node.conn()
    #     self.node.set_task_result_obj()

    # def down_detail(self, standardid):
    #     url = "http://www.wanfangdata.com.cn/details/detail.do?_type=techResult&id={}".format(standardid)
    #     BoolResult, errString, r = facade.BaseRequest(url, mark="title")
    #     if BoolResult:
    #         downdate = MDateTime().get_today_date()
    #         jsondicts = {}
    #         jsondicts["downdate"] = downdate
    #         jsondicts["html"] = r.text
    #         jsonmsg = json.dumps(jsondicts, ensure_ascii=False)
    #         filepath = BaseDir.get_new_path(sPath, downdate + ".big_json")
    #         BaseDir.single_add_file(filepath, jsonmsg + "\n")
    #         return True
    #     else:
    #         print("下载出错，请检查")
    #         return False
    #
    # def task(self):
    #     while True:
    #         sql = "select standardid from articlenew where stat=0 and failcount<3 limit 1000"
    #         rows = self.mysqlutils.SelectFromDB(sql)
    #         if rows:
    #             for row in rows:
    #                 time.sleep(1)
    #                 if self.down_detail(row[0]):
    #                     sql = "update articlenew set stat=1 where standardid='{}'".format(row[0])
    #                     self.mysqlutils.ExeSqlToDB(sql)
    #                 else:
    #                     sql = "update articlenew set failcount=failcount+1 where standardid='{}'".format(row[0])
    #                     self.mysqlutils.ExeSqlToDB(sql)
    #         else:
    #             break

    def proxyIpFromDB(self):
        result = getDataFromRedis(curpath, "db.ini")
        ProxyPool = list(result)
        return ProxyPool

    def getTasksFromDB(self):
        sql = "select standardid from articlenew where stat=0 and failcount<3 limit 1000"
        rows = self.mysqlutils.SelectFromDB(sql)
        return rows

    def down_detail(self, threadval, ringproxy, standardid):
        result_queue = threadval.result_queue
        url = "http://www.wanfangdata.com.cn/details/detail.do?_type=techResult&id={}".format(standardid)
        proxy = ringproxy.get_random()
        proxys = {
            "http": proxy,
            "https": proxy
        }
        print(proxys)
        self.headers["X-Forwarded-For"] = proxy.split(":")[0]
        print(self.headers["X-Forwarded-For"])
        BoolResult, errString, r = facade.BaseRequest(url,
                                                      headers=self.headers,
                                                      proxies=proxys,
                                                      mark="title",
                                                      timeout=(10, 20))
        if BoolResult:
            downdate = MDateTime().get_today_date()
            jsondicts = {}
            jsondicts["downdate"] = downdate
            jsondicts["html"] = r.text
            jsonmsg = json.dumps(jsondicts, ensure_ascii=False)
            outPathFile = os.path.join(sPath,
                                       '%s_%s_%d_%d.big_json' % (
                                           downdate, GetLocalIPByPrefix('192.168.'), os.getpid(),
                                           threading.get_ident())
                                       )
            print('Write to %s ...' % outPathFile)
            BaseDir.single_add_file(outPathFile, jsonmsg + "\n")
            result_queue.put((standardid, 1))
        else:
            print("下载出错，请检查")
            if r:
                BaseDir.single_write_file(r.text, "./test.html")
            ringproxy.remove(proxy)
            result_queue.put((standardid, 0))
            return False

    def InsertIntoDbFromList(self, results):
        for result in results:
            if result[1] == 1:
                sql = "update articlenew set stat=1 where standardid='{}'".format(result[0])
                self.mysqlutils.ExeSqlToDB(sql)
            else:
                sql = "update articlenew set failcount=failcount+1 where standardid='{}'".format(result[0])
                self.mysqlutils.ExeSqlToDB(sql)


class DetailThreadRun(MThreadingRun):
    def __init__(self, num):
        self.down = DownDetail()
        super(DetailThreadRun, self).__init__(num)

    def getTask(self, *args, **kwargs):
        results = self.down.getTasksFromDB()
        return results

    def setTask(self, results=None, *args, **kwargs):
        for url_tasks in results:
            # 将每一页加入任务队列
            self.add_job(self.func, url_tasks[0])

    def dealresult(self, *args, **kwargs):
        self.down.InsertIntoDbFromList(self.results)

    def setProxy(self, proxysList=None):
        # rows = self.down.proxyIpFromDB()
        MThreadingRun.setProxy(self, self.down.proxysdicts)
        time.sleep(30)

    def is_break(self):
        return True

    def thread_pool_hook(self, thread_pool_dicts, thread, args, kwargs):
        return {}

    def fun(self, threadval, *args, **kwargs):
        standardid = args[0]
        self.down.down_detail(threadval, self.list_proxy, standardid)


if __name__ == "__main__":
    down = DetailThreadRun(40)
    down.run(model=1)
